/**
 * Copyright 2012 University of Massachusetts Amherst
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *   http://www.apache.org/licenses/LICENSE-2.0
 *   
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
*/
package com.googlecode.clearnlp.tokenization;

import java.io.BufferedReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import com.googlecode.clearnlp.util.pair.Pair;
import com.googlecode.clearnlp.util.pair.StringBooleanPair;

/**
 * @since 1.1.0
 * @author Jinho D. Choi ({@code jdchoi77@gmail.com})
 */
abstract public class AbstractTokenizer implements Tokenizer
{
	/* (non-Javadoc)
	 * @see com.googlecode.clearnlp.tokenization.Tokenizer#getTokens(java.io.BufferedReader)
	 */
	@Override
	public List<Token> getTokens(BufferedReader fin)
	{
		List<Token> tokens = new ArrayList<Token>();
		String line;
		
		try
		{
			while ((line = fin.readLine()) != null)
				tokens.addAll(getTokens(line.trim()));
		}
		catch (IOException e) {e.printStackTrace();}
		
		return tokens;
	}
	
	/* (non-Javadoc)
	 * @see com.googlecode.clearnlp.tokenization.Tokenizer#getTokens(java.lang.String)
	 */
	@Override
	public List<Token> getTokens(String str)
	{
		List<FlaggedToken> lTokens = getTokenList(str);
		List<Token> tokens = new ArrayList<Token>(lTokens.size());
		
		for (FlaggedToken fTok : lTokens)
			tokens.add(fTok.tok);
		
		return tokens;
	}
	
	abstract public List<FlaggedToken> getTokenList(String str);
}

